/*
 * � The National Archives 2005-2006.  All rights reserved.
 * See Licence.txt for full licence details.
 *
 * Developed by:
 * Tessella Support Services plc
 * 3 Vineyard Chambers
 * Abingdon, OX14 3PX
 * United Kingdom
 * http://www.tessella.com
 *
 * Tessella/NPD/4305
 * PRONOM 4
 *
 * $Id: AnalysisController.java,v 1.15 2006/03/13 15:15:25 linb Exp $
 *
 * $Log: AnalysisController.java,v $
 * Revision 1.15  2006/03/13 15:15:25  linb
 * Changed copyright holder from Crown Copyright to The National Archives.
 * Added reference to licence.txt
 * Changed dates to 2005-2006
 *
 * Revision 1.14  2006/02/13 11:27:01  linb
 * - Correct spelling in error message
 * - Fix </Version> tag
 *
 * Revision 1.13  2006/02/09 15:31:23  linb
 * Updates to javadoc and code following the code review
 *
 * Revision 1.12  2006/02/09 13:15:46  linb
 * Removed un-used class fileReader
 *
 * Revision 1.11  2006/02/09 12:14:05  linb
 * Updated version number to 1.1
 * Changed some javadoc to allow it to be created cleanly
 *
 * Revision 1.10  2006/02/08 14:09:24  linb
 * - Updated namespaces to all point to nationalarchives.gov.uk
 *
 * Revision 1.9  2006/02/08 11:44:50  linb
 * - make saveConfiguration throw an IOException
 *
 * Revision 1.8  2006/02/07 17:16:22  linb
 * - Change fileReader to ByteReader in formal parameters of methods
 * - use new static constructors
 * - Add detection of if a filePath is a URL or not
 *
 * Revision 1.7  2006/02/07 10:52:34  linb
 * - Altered FileCollection input to be able to use elements (instead of attributes) for most of the data
 * - Altered output routine to do the same
 *
 * Revision 1.6  2006/01/31 16:47:29  linb
 * Added log messages that were missing due to the log keyword being added too late
 *
 * Revision 1.5  2006/01/31 16:21:20  linb
 * Removed the dollars from the log lines generated by the previous message, so as not to cause problems with subsequent commits
 *
 * Revision 1.4  2006/01/31 16:19:07  linb
 * Added Log: and Id: tags to these files
 *
 * Revision 1.3  2006/01/31 16:11:37  linb
 * Add support for XML namespaces to:
 * 1) The reading of the config file, spec file and file-list file
 * 2) The writing of the config file and file-list file
 * - The namespaces still need to be set to their proper URIs (currently set to example.com...)
 * - Can still read in files without namespaces
 *
 * Revision 1.2  2006/01/31 12:00:37  linb
 * - Added new text field to option dialog for proxy setting
 * - Added new get/set methods to AnalysisController for proxy settings (from ConfigFile) *
 *
 * $History: AnalysisController.java $
 * 
 * *****************  Version 53  *****************
 * User: Walm         Date: 20/10/05   Time: 15:17
 * Updated in $/PRONOM4/FFIT_SOURCE
 * allow connection to web service through a proxy.  
 * Pass on proxy connection details to web service.
 * When web service connection fails, give an appropriate warning to the
 * user.
 * 
 * *****************  Version 52  *****************
 * User: Walm         Date: 7/06/05    Time: 12:29
 * Updated in $/PRONOM4/FFIT_SOURCE
 * Control the writing to file list explicitly so that it is not done in
 * full in memory prior to writing out to file.
 *
 * *****************  Version 51  *****************
 * User: Walm         Date: 6/06/05    Time: 11:46
 * Updated in $/PRONOM4/FFIT_SOURCE
 * Ensure XML files are saved in UTF8
 * Resolve JIRA bug PRON-15
 *
 * *****************  Version 50  *****************
 * User: Walm         Date: 17/05/05   Time: 12:50
 * Updated in $/PRONOM4/FFIT_SOURCE
 * update to version 1.0.7
 *
 * *****************  Version 49  *****************
 * User: Mals         Date: 12/05/05   Time: 12:39
 * Updated in $/PRONOM4/FFIT_SOURCE
 * Updated Version number to V1.0.6
 *
 * *****************  Version 48  *****************
 * User: Walm         Date: 12/05/05   Time: 10:15
 * Updated in $/PRONOM4/FFIT_SOURCE
 * Version number changed to 1.0.5
 *
 * *****************  Version 47  *****************
 * User: Walm         Date: 10/05/05   Time: 19:42
 * Updated in $/PRONOM4/FFIT_SOURCE
 * update release version
 *
 * *****************  Version 46  *****************
 * User: Walm         Date: 10/05/05   Time: 19:23
 * Updated in $/PRONOM4/FFIT_SOURCE
 * Update default configuration values
 *
 * *****************  Version 45  *****************
 * User: Walm         Date: 10/05/05   Time: 19:20
 * Updated in $/PRONOM4/FFIT_SOURCE
 * show warning if web service fails to get signature file version
 *
 * *****************  Version 44  *****************
 * User: Mals         Date: 9/05/05    Time: 14:47
 * Updated in $/PRONOM4/FFIT_SOURCE
 * Changed version number to V1.0.2
 *
 * *****************  Version 43  *****************
 * User: Walm         Date: 5/05/05    Time: 10:28
 * Updated in $/PRONOM4/FFIT_SOURCE
 * save to file at end of run is now incorporated into
 * setAnalysisComplete()
 *
 * *****************  Version 42  *****************
 * User: Mals         Date: 4/05/05    Time: 10:12
 * Updated in $/PRONOM4/FFIT_SOURCE
 * Changed version number to V1.0.1
 *
 * *****************  Version 41  *****************
 * User: Walm         Date: 26/04/05   Time: 17:29
 * Updated in $/PRONOM4/FFIT_SOURCE
 * show different messages in GUI and on command line
 *
 * *****************  Version 40  *****************
 * User: Walm         Date: 25/04/05   Time: 16:33
 * Updated in $/PRONOM4/FFIT_SOURCE
 * added updateDateLastDownload to allow the configuration file to be
 * updated with a new DateLastDownload whenever the user checks for a
 * newer signature file (even if none is found)
 *
 * *****************  Version 39  *****************
 * User: Mals         Date: 20/04/05   Time: 12:16
 * Updated in $/PRONOM4/FFIT_SOURCE
 * +Saves date in XML in format yyyy-MM-ddTHH:mm:ss
 * +Displays all dates in format  dd-MMM-yyyy
 *
 * *****************  Version 38  *****************
 * User: Walm         Date: 19/04/05   Time: 18:25
 * Updated in $/PRONOM4/FFIT_SOURCE
 * Update default configuration values to reflect name change from uk to
 * DROID. + Make code cope better with badly formed XML
 *
 * *****************  Version 37  *****************
 * User: Walm         Date: 19/04/05   Time: 15:35
 * Updated in $/PRONOM4/FFIT_SOURCE
 * bug correction: unable to save list to file if PUID is null
 *
 * *****************  Version 36  *****************
 * User: Mals         Date: 19/04/05   Time: 9:41
 * Updated in $/PRONOM4/FFIT_SOURCE
 * Tessella Ref: NPD/4305/PR/IM/2005APR18/09:51:03
 * Issues
 * ----------
 * 36. wording of the Status and Warning texts should be amended in line
 * with that given in a previous email
 * (NPD/4305/CL/CSC/2005FEB17/16:34:13)
 *
 * +Any  to FFITVersion changed to DROIDVersion when writing XML file
 *
 *
 * *****************  Version 35  *****************
 * User: Mals         Date: 13/04/05   Time: 10:24
 * Updated in $/PRONOM4/FFIT_SOURCE
 * Ref:Email from A.Brown NPD/4305/CL/CSC/2005APR12/13:11  File ID GUI
 * Export to CSV writes DROID Version instead of uk
 *
 * *****************  Version 34  *****************
 * User: Mals         Date: 7/04/05    Time: 16:39
 * Updated in $/PRONOM4/FFIT_SOURCE
 * Add application version, signature file version and date to csv export
 *
 * *****************  Version 33  *****************
 * User: Walm         Date: 7/04/05    Time: 14:43
 * Updated in $/PRONOM4/FFIT_SOURCE
 * review headers
 *
 * *****************  Version 32  *****************
 * User: Walm         Date: 5/04/05    Time: 18:08
 * Updated in $/PRONOM4/FFIT_SOURCE
 * move signature file parser to FFSignatureFile
 *
 * *****************  Version 31  *****************
 * User: Walm         Date: 5/04/05    Time: 14:54
 * Updated in $/PRONOM4/FFIT_SOURCE
 * increase size of random access reader buffer
 *
 * *****************  Version 30  *****************
 * User: Walm         Date: 4/04/05    Time: 17:44
 * Updated in $/PRONOM4/FFIT_SOURCE
 * move saveConfig code to ConfigFile class
 *
 * *****************  Version 29  *****************
 * User: Mals         Date: 1/04/05    Time: 10:26
 * Updated in $/PRONOM4/FFIT_SOURCE
 * +When saving a file list(without results) make sure warnings are blank
 *
 * *****************  Version 28  *****************
 * User: Walm         Date: 31/03/05   Time: 15:25
 * Updated in $/PRONOM4/FFIT_SOURCE
 * Add some constants for default configuration settings
 * + add option to not read in signature file when downloading it (used in
 * command line mode)
 * + add option to not save configuration settings when (used in command
 * line mode since this does not use a configuration file)
 *
 * *****************  Version 27  *****************
 * User: Mals         Date: 30/03/05   Time: 17:12
 * Updated in $/PRONOM4/FFIT_SOURCE
 * +Doesn't save identification status when saving a file list
 *
 * *****************  Version 26  *****************
 * User: Mals         Date: 30/03/05   Time: 16:41
 * Updated in $/PRONOM4/FFIT_SOURCE
 * +File list can be saved without saving the results
 *
 * *****************  Version 25  *****************
 * User: Mals         Date: 30/03/05   Time: 15:42
 * Updated in $/PRONOM4/FFIT_SOURCE
 * +Export to CSV
 * +return methods for config settings
 *
 * *****************  Version 24  *****************
 * User: Walm         Date: 30/03/05   Time: 9:34
 * Updated in $/PRONOM4/FFIT_SOURCE
 * download signature file and save config file
 *
 * *****************  Version 23  *****************
 * User: Walm         Date: 29/03/05   Time: 16:55
 * Updated in $/PRONOM4/FFIT_SOURCE
 * Deal with errors in XML file definitions when reading them in
 *
 * *****************  Version 22  *****************
 * User: Walm         Date: 29/03/05   Time: 12:04
 * Updated in $/PRONOM4/FFIT_SOURCE
 * check that signature file in loaded file is identical to the current
 * one for the application
 *
 * *****************  Version 21  *****************
 * User: Walm         Date: 29/03/05   Time: 11:04
 * Updated in $/PRONOM4/FFIT_SOURCE
 * Update the XML definition for the file hit collection file
 *
 * *****************  Version 20  *****************
 * User: Walm         Date: 24/03/05   Time: 11:18
 * Updated in $/PRONOM4/FFIT_SOURCE
 * reformat fileCollection file
 * + functionality to check the latest signature file version on PRONOM
 * web service
 *
 * *****************  Version 19  *****************
 * User: Mals         Date: 24/03/05   Time: 9:35
 * Updated in $/PRONOM4/FFIT_SOURCE
 * Saving collection as XML adds hit file format information
 *
 * *****************  Version 18  *****************
 * User: Walm         Date: 18/03/05   Time: 12:41
 * Updated in $/PRONOM4/FFIT_SOURCE
 * make sure all if statements use curly brackets
 *
 * *****************  Version 17  *****************
 * User: Walm         Date: 17/03/05   Time: 17:41
 * Updated in $/PRONOM4/FFIT_SOURCE
 * Add a constant for buffer size for reading random access files
 *
 * *****************  Version 16  *****************
 * User: Mals         Date: 16/03/05   Time: 16:31
 * Updated in $/PRONOM4/FFIT_SOURCE
 * Added SaveFileList
 *
 * *****************  Version 15  *****************
 * User: Walm         Date: 16/03/05   Time: 10:29
 * Updated in $/PRONOM4/FFIT_SOURCE
 * AnalysisThread class now has access to AnalysisController and so can
 * call non-static methods
 *
 * *****************  Version 14  *****************
 * User: Walm         Date: 16/03/05   Time: 9:52
 * Updated in $/PRONOM4/FFIT_SOURCE
 * Make static/non static methods more consistent.
 * The main method now either launches GUI or command line controller
 *
 * *****************  Version 13  *****************
 * User: Mals         Date: 15/03/05   Time: 15:14
 * Updated in $/PRONOM4/FFIT_SOURCE
 * Added remove file by index method
 *
 * *****************  Version 12  *****************
 * User: Walm         Date: 15/03/05   Time: 10:54
 * Updated in $/PRONOM4/FFIT_SOURCE
 * reorder file classification constants (to help ordering by file status
 * in GUI)
 *
 * *****************  Version 11  *****************
 * User: Walm         Date: 14/03/05   Time: 17:31
 * Updated in $/PRONOM4/FFIT_SOURCE
 * Ability to cancel an analysis part way through, and to query progress
 * more efficiently
 *
 * *****************  Version 10  *****************
 * User: Walm         Date: 14/03/05   Time: 15:58
 * Updated in $/PRONOM4/FFIT_SOURCE
 * undo some temporary debugging code
 *
 * *****************  Version 9  *****************
 * User: Walm         Date: 14/03/05   Time: 15:56
 * Updated in $/PRONOM4/FFIT_SOURCE
 * Implements analysis in a separate thread
 *
 * *****************  Version 8  *****************
 * User: Mals         Date: 14/03/05   Time: 14:31
 * Updated in $/PRONOM4/FFIT_SOURCE
 * runFileIdentification accepts IdentificationFile parameter
 *
 * *****************  Version 7  *****************
 * User: Mals         Date: 14/03/05   Time: 14:01
 * Updated in $/PRONOM4/FFIT_SOURCE
 * Launches GUI on startup
 *
 * *****************  Version 6  *****************
 * User: Mals         Date: 14/03/05   Time: 9:47
 * Updated in $/PRONOM4/FFIT_SOURCE
 * Launches GUI from main method
 */
package uk.gov.nationalarchives.droid;

import java.io.FileInputStream;
import java.io.IOException;
import java.net.URL;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.Iterator;

import javax.xml.parsers.SAXParser;
import javax.xml.parsers.SAXParserFactory;

import org.jdom.Element;
import org.jdom.output.XMLOutputter;
import org.xml.sax.InputSource;
import org.xml.sax.XMLReader;

import uk.gov.nationalarchives.droid.GUI.FileIdentificationPane;
import uk.gov.nationalarchives.droid.binFileReader.ByteReader;
import uk.gov.nationalarchives.droid.commandLine.CmdController;
import uk.gov.nationalarchives.droid.signatureFile.FFSignatureFile;
import uk.gov.nationalarchives.droid.xmlReader.PronomWebService;
import uk.gov.nationalarchives.droid.xmlReader.SAXModelBuilder;
import uk.gov.nationalarchives.droid.stats.StatsLogger;
import uk.gov.nationalarchives.droid.stats.StatsExporterFactory;
import uk.gov.nationalarchives.droid.stats.StatsExportFormat;

/**
 * Controls the file format identification analysis
 * On startup, it launches either the command line or the GUI package.
 * It then exposes the necessary methods for running the file
 * identification analysis.
 *
 * @author Martin Waller
 * @version 1.1.0
 */
public class AnalysisController {
    //Application version
    public static final String DROID_VERSION = "3.0";

    //File classification constants
    public static final int FILE_CLASSIFICATION_POSITIVE = 1;
    public static final int FILE_CLASSIFICATION_TENTATIVE = 2;
    public static final int FILE_CLASSIFICATION_NOHIT = 3;
    public static final int FILE_CLASSIFICATION_ERROR = 4;
    public static final int FILE_CLASSIFICATION_NOTCLASSIFIED = 5;
    public static final String FILE_CLASSIFICATION_POSITIVE_TEXT = "Positive";
    public static final String FILE_CLASSIFICATION_TENTATIVE_TEXT = "Tentative";
    public static final String FILE_CLASSIFICATION_NOHIT_TEXT = "Not identified";
    public static final String FILE_CLASSIFICATION_ERROR_TEXT = "Error";
    public static final String FILE_CLASSIFICATION_NOTCLASSIFIED_TEXT = "Not yet run";

    //hit type constants
    public static final int HIT_TYPE_POSITIVE_SPECIFIC = 10;
    public static final int HIT_TYPE_POSITIVE_GENERIC = 11;
    public static final int HIT_TYPE_TENTATIVE = 12;
    public static final int HIT_TYPE_POSITIVE_GENERIC_OR_SPECIFIC = 15;
    public static final String HIT_TYPE_POSITIVE_SPECIFIC_TEXT = "Positive (Specific Format)";
    public static final String HIT_TYPE_POSITIVE_GENERIC_TEXT = "Positive (Generic Format)";
    public static final String HIT_TYPE_TENTATIVE_TEXT = "Tentative";
    public static final String HIT_TYPE_POSITIVE_GENERIC_OR_SPECIFIC_TEXT = "Positive";

    //Buffer size for reading random access files
    public static final int FILE_BUFFER_SIZE = 100000000;

    //default values
    public static final int CONFIG_DOWNLOAD_FREQ = 30;
    public static final String CONFIG_FILE_NAME = "DROID_config.xml";
    public static final String FILE_LIST_FILE_NAME = "DROID_filecollection.xml";
    public static final String PRONOM_WEB_SERVICE_URL = "http://www.nationalarchives.gov.uk/pronom/service.asmx";
    public static final String SIGNATURE_FILE_NAME = "DROID_SignatureFile.xml";
    public static final String PUID_RESOLUTION_URL = "http://www.nationalarchives.gov.uk/pronom/";
    public static final String BROWSER_PATH = "/usr/bin/firefox";
    public static final String LABEL_APPLICATION_VERSION = "DROIDVersion";
    public static final String LABEL_DATE_CREATED = "DateCreated";

    /**
     * Date format to read/write dates to XML
     */
    private static final String XML_DATE_FORMAT = "yyyy'-'MM'-'dd'T'HH:mm:ss";

    /**
     * Date format to display dates in application
     */
    private static final String DISPLAY_DATE_FORMAT = "dd'-'MMM'-'yyyy";

    /**
     * Namespace for the xml file collection file
     */
    public static final String FILE_COLLECTION_NS = "http://www.nationalarchives.gov.uk/pronom/FileCollection";
    /**
     * Namespace for the xml configuration file
     */
    public static final String CONFIG_FILE_NS = "http://www.nationalarchives.gov.uk/pronom/ConfigFile";
    /**
     * Namespace for the xml file format signatures file
     */
    public static final String SIGNATURE_FILE_NS = "http://www.nationalarchives.gov.uk/pronom/SignatureFile";

    //class variables:
    private ConfigFile myConfigFile = new ConfigFile();
    private FileCollection myFileCollection = new FileCollection();
    private FFSignatureFile mySigFile;
    private boolean myAnalysisCancelled = false;
    private boolean verbose = true;
    private int myNumCompletedFiles = 0;
    private boolean isAnalysisRunning = false;
    // Time that profiling / analysis started and finished
    private Date startTime;
    private Date completedTime;
    
    /**
     * Statistics logger
     */
    StatsLogger myLogger;
    
    /**
     * output formats to be used for saving results at end of run
     */
    private String myOutFormats = "";
    /**
     * base file name to be used for saving results at end of run
     */
    private String myOutFileName = "";

    /**
     * Latest version of signature file available , value is -1 if this hasn't been checked
     */
    private int sigFileLatestVersion = -1;

    public AnalysisController() {
        myFileCollection = new FileCollection();
    }

    /**
     * Launches an instance of uk either with a GUI interface or in a command line environment,
     * depending on the run time arguments.
     *
     * @param args The run time arguments
     */
    public static void main(String[] args) throws Exception {

        if (isGUI(args)) {
            FileIdentificationPane.launch(new AnalysisController());
        } else if (isCmdLine(args)) {
            CmdController myCmdController;
            myCmdController = new CmdController(args);
        }

    }

    /**
     * Launch the statistics thread on the files that have been listed and using
     * the signature file that has been opened.  Save results to file at the end of the run.
     *
     * @param theOutFormats  string containing formats for the output (code looks for CSV and XML in the string)
     * @param theOutFileName name of file to which to save results at end of run
     * @return pointer to StatsThread
     */
    public StatsThread runStatsGathering(String theOutFormats, String theOutFileName) {
                
        myAnalysisCancelled = false;
        myOutFileName = theOutFileName;
        myOutFormats = theOutFormats;                
        myLogger = new StatsLogger(myFileCollection.getNumFiles());
        
        StatsThread thread = new StatsThread(myLogger,myFileCollection, mySigFile, this);
        thread.start();
        return thread;
    }  

    /**
     * Launch the statistics thread on the array of filenames provided and using
     * the signature file that has been opened.  Save results to file at the end of the run.
     *
     * @param theOutFormats  string containing formats for the output (code looks for CSV and XML in the string)
     * @param theOutFileName name of file to which to save results at end of run
     * @param files array of filenames to run stats on
     * @return pointer to StatsThread
     */
    public StatsThread runStatsGathering(String theOutFormats, String theOutFileName, String[] files, boolean isRecursive) {
        
        myAnalysisCancelled = false;
        myOutFileName = theOutFileName;
        myOutFormats = theOutFormats;
        myLogger = new StatsLogger(myFileCollection.getNumFiles());
        
        StatsThread thread = new StatsThread(myLogger, files, mySigFile, this, isRecursive);
        thread.start();
        return thread;
    }    
    
    /*
    *Created a date object with value of date in format yyyy-MM-ddTHH:mm:ss (e.g. 2005-02-24T12:35:23)
    * @param   XMLFormatDate   date in format yyyy-MM-ddTHH:mm:ss
    * @return  date with value set
    */
    public static java.util.Date parseXMLDate(String XMLFormatDate) throws java.text.ParseException {
        SimpleDateFormat xmlDateFormat = new SimpleDateFormat(XML_DATE_FORMAT);
        try {
            //parse the date supplied  into a Date object
            return xmlDateFormat.parse(XMLFormatDate);

        } catch (java.text.ParseException pe) {
            throw pe;
        }
    }

    /**
     * Creates an XML format date yyyy-MM-ddTHH:mm:ss from a date object.
     * <p/>
     * For example, 2005-02-24T12:35:23
     *
     * @param aDate Date to represent
     * @return Date in formatyyyy-MM-ddTHH:mm:ss (e.g. 2005-02-24T12:35:23)
     */
    public static String writeXMLDate(java.util.Date aDate) {
        SimpleDateFormat xmlDateFormat = new SimpleDateFormat(XML_DATE_FORMAT);
        return xmlDateFormat.format(aDate);
    }

    /**
     * Creates a date in format dd-MMM-yyyy (e.g 18-Nov-2005)
     *
     * @param aDate Date to represent
     * @return Date in format dd-MMM-yyyy (e.g 18-Nov-2005)
     */
    public static String writeDisplayDate(java.util.Date aDate) {
        SimpleDateFormat displayDateFormat = new SimpleDateFormat(DISPLAY_DATE_FORMAT);
        return displayDateFormat.format(aDate);
    }

    /**
     * Displays the contents of the binary file as a list of bytes.
     * This is only used for debugging purposes.
     *
     * @param testFile The binary file
     */
    private void debugDisplayBinFile(ByteReader testFile) {

        System.out.println("======================= " + testFile.getFileName());

        //printout testfile for debugging
        String allTheBytes = "";

        //for(int i = 0; i<byteToTest.length; i++) {
        for (int i = 0; i < 200; i++) {
            allTheBytes += showByte(testFile.getByte(i));
        }

        System.out.println(allTheBytes);

    }


    /**
     * Displays a byte in the same format as used by TNA.
     *
     * @param theByte The byte to display
     */
    private String showByte(byte theByte) {
        String byteDisplay;
        byteDisplay = Integer.toHexString(theByte);
        if (byteDisplay.length() == 1) {
            byteDisplay = "0" + byteDisplay;
        }
        if (byteDisplay.length() > 2) {
            byteDisplay = byteDisplay.substring(byteDisplay.length() - 2);
        }
        return byteDisplay;
    }


    /**
     * Determines whether to launch uk in GUI mode
     *
     * @param args The run time arguments
     */
    private static boolean isGUI(String[] args) {
        if (args.length == 0) {
            return true;
        } else {
            return false;
        }
    }

    /**
     * Determines whether to launch uk in command line mode
     *
     * @param args The run time arguments
     */
    private static boolean isCmdLine(String[] args) {
        if (args.length > 0) {
            return true;
        } else {
            return false;
        }
    }

    /**
     * Reads the default configuration file, and loads the contents into memory.
     */
    public void readConfiguration() throws Exception {
        readConfiguration(CONFIG_FILE_NAME);  //use default file name
    }


    /**
     * Config file is URL makes tech watch wrapper easier to use
     *
     * @param configFileURL
     * @throws Exception
     */
    public void readConfiguration(URL configFileURL) throws Exception {
        try {
            myConfigFile = new ConfigFile();

            SAXParserFactory factory = SAXParserFactory.newInstance();
            factory.setNamespaceAware(true);
            SAXParser saxParser = factory.newSAXParser();
            XMLReader parser = saxParser.getXMLReader();
            SAXModelBuilder mb = new SAXModelBuilder();
            mb.setObjectPackage("uk.gov.nationalarchives.droid");
            mb.setupNamespace(CONFIG_FILE_NS, true);
            parser.setContentHandler(mb);

            //read in the XML file
            java.io.BufferedReader in = new java.io.BufferedReader(new java.io.InputStreamReader(configFileURL.openStream()));
            parser.parse(new InputSource(in));
            myConfigFile = (ConfigFile) mb.getModel();
            myConfigFile.setFileName(configFileURL.getFile());
        } catch (Exception e) {
            throw new Exception("Unable to read configuration file " + configFileURL.getFile() + "\nThe following error was encountered: " + e.toString());
        }

    }

    /**
     * Reads a configuration file, and loads the contents into memory.
     *
     * @param theFileName The name of the configuration file to open
     */
    public void readConfiguration(String theFileName) throws Exception {

        //if file doesn't exist, then warn user and create a new one using defaults
        if (!isFileFound(theFileName)) {
            MessageDisplay.generalWarning("The expected configuration file " + theFileName + " was not found.\nA new one will be created using the configuration defaults.");
            myConfigFile = new ConfigFile();
            try {
                saveConfiguration(theFileName);
            } catch (IOException e) {
                MessageDisplay.generalWarning("Unable to save configuration updates");
            }
        } else {
            try {

                checkFile(theFileName);

                //prepare for XML read
                MessageDisplay.resetXMLRead();

                //prepare to read in the XML file
                SAXParserFactory factory = SAXParserFactory.newInstance();
                factory.setNamespaceAware(true);
                SAXParser saxParser = factory.newSAXParser();
                XMLReader parser = saxParser.getXMLReader();
                SAXModelBuilder mb = new SAXModelBuilder();
                mb.setObjectPackage("uk.gov.nationalarchives.droid");
                mb.setupNamespace(CONFIG_FILE_NS, true);
                parser.setContentHandler(mb);

                //read in the XML file
                java.io.BufferedReader in = new java.io.BufferedReader(new java.io.InputStreamReader(new java.io.FileInputStream(theFileName), "UTF8"));
                parser.parse(new InputSource(in));
                myConfigFile = (ConfigFile) mb.getModel();
                myConfigFile.setFileName(theFileName);

                //let the user know the outcome if there were any problems
                int numXMLWarnings = MessageDisplay.getNumXMLWarnings();
                if (numXMLWarnings > 0) {
                    String successMessage = "The configuration file " + theFileName;
                    successMessage += " contained " + numXMLWarnings + " warning(s)";
                    MessageDisplay.generalWarning(successMessage);
                }

            } catch (Exception e) {
                throw new Exception("Unable to read configuration file " + theFileName + "\nThe following error was encountered: " + e.toString());
                //messageDisplay.generalWarning(e.toString());
                //System.exit(0);
            }
        }
    }

    /**
     * Saves the current configuration to the default file name
     */
    public void saveConfiguration() throws IOException {
        myConfigFile.saveConfiguration();
    }

    /**
     * Saves the current configuration to file in XML format
     *
     * @param filePath Path of configuration file
     */
    public void saveConfiguration(String filePath) throws IOException {
        myConfigFile.setFileName(filePath);
        myConfigFile.saveConfiguration();
    }

    /**
     * Saves the file list to file in XML format with or without the associated hits
     *
     * @param filePath    Path of where to save file list
     * @param saveResults Save the file format hits as well
     */
    public void saveFileList(String filePath, boolean saveResults) {

        try {
            java.io.BufferedWriter out = new java.io.BufferedWriter(
                    new java.io.OutputStreamWriter(new java.io.FileOutputStream(filePath), "UTF8"));
            writeXmlWithElements(out, saveResults);
        } catch (java.io.IOException e) {
        }
    }

    /**
     * Write the XML to the file, using the new schema format with elements for most of the data.
     */
    private void writeXmlWithElements(final java.io.BufferedWriter out, final boolean saveResults) throws IOException {

        out.write("<?xml version=\"1.0\" encoding=\"UTF-8\"?>");
        out.newLine();
        out.write("<FileCollection xmlns=\"" + FILE_COLLECTION_NS + "\">");
        out.newLine();
        out.write("  <DROIDVersion>" + AnalysisController.getDROIDVersion().replaceAll("&", "&amp;") + "</DROIDVersion>");
        out.newLine();
        out.write("  <SignatureFileVersion>" + Integer.toString(this.getSigFileVersion()) + "</SignatureFileVersion>");
        out.newLine();
        out.write("  <DateCreated>" + writeXMLDate(new java.util.Date()) + "</DateCreated>");
        out.newLine();

        //loop through file objects
        Iterator<IdentificationFile> fileIterator = this.getFileIterator();
        IdentificationFile idFile;
        while (fileIterator.hasNext()){

            idFile = fileIterator.next();

            //create IdentificationFile element and its attributes
            out.write("  <IdentificationFile ");
            if (saveResults) {
                out.write("IdentQuality=\"" + idFile.getClassificationText() + "\" ");
            }
            out.write(">");
            out.newLine();
            out.write("    <FilePath>" + idFile.getFilePath().replaceAll("&", "&amp;") + "</FilePath>");
            out.newLine();
            if (saveResults && !"".equals(idFile.getWarning())) {
                out.write("    <Warning>" + idFile.getWarning().replaceAll("&", "&amp;") + "</Warning>");
                out.newLine();
            }

            //Add file format hits if required to do so
            if (saveResults) {
                //now create an FileFormatHit element for each hit
                for (int hitCounter = 0; hitCounter < idFile.getNumHits(); hitCounter++) {

                    FileFormatHit formatHit = idFile.getHit(hitCounter);
                    out.write("    <FileFormatHit>");
                    out.newLine();
                    out.write("      <Status>" + formatHit.getHitTypeVerbose() + "</Status>");
                    out.newLine();
                    out.write("      <Name>" + formatHit.getFileFormatName().replaceAll("&", "&amp;") + "</Name>");
                    out.newLine();
                    if (formatHit.getFileFormatVersion() != null) {
                        out.write("      <Version>" + formatHit.getFileFormatVersion().replaceAll("&", "&amp;") + "</Version>");
                        out.newLine();
                    }
                    if (formatHit.getFileFormatPUID() != null) {
                        out.write("      <PUID>" + formatHit.getFileFormatPUID().replaceAll("&", "&amp;") + "</PUID>");
                        out.newLine();
                    }
                    if (formatHit.getMimeType() != null) {
                        out.write("      <MimeType>" + formatHit.getMimeType().replaceAll("&", "&amp;") + "</MimeType>");
                        out.newLine();
                    }
                    if (!"".equals(formatHit.getHitWarning())) {
                        out.write("      <IdentificationWarning>"
                                + formatHit.getHitWarning().replaceAll("&", "&amp;") + "</IdentificationWarning>");
                        out.newLine();
                    }
                    out.write("    </FileFormatHit>");
                    out.newLine();
                }//end file hit FOR

            }//end if (saveResults)

            //close IdentificationFile element
            out.write("  </IdentificationFile>");
            out.newLine();

        }//end idFile While

        //close FileCollection element
        out.write("</FileCollection>");
        out.newLine();

        out.flush();
        out.close();
    }


    /**
     * Writes the file collection to a CSV file
     *
     * @param filePath where to save the CSV file
     */
    public void exportFileCollectionAsCSV(String filePath) {
        StringBuffer fileText = new StringBuffer();

        //Initialise loop variables
        int fileCounter = 0;
        int hitCounter = 0;
        FileFormatHit hit = null;

        //Write uk Version , Signature file version and date at top

        fileText.append("DROID Version,\"" + getDROIDVersion() + "\"");
        fileText.append(",,SigFile Version,\"" + getSigFileVersion() + "\"");
        fileText.append(",,Date Created,\"" + writeDisplayDate(new Date()) + "\"");
        fileText.append("\n");

        //Write column headers
        fileText.append("Status,File,Warning,PUID,MIME,Format,Version,Status,Warning\n");
        
        //Iterate through IdentifactionFile objects in file collection
        IdentificationFile idFile;
        Iterator<IdentificationFile> fileIterator = this.getFileIterator();
        
        while (fileIterator.hasNext()){

            idFile = fileIterator.next();
            
            //Write the identification classification , file name and warning(if it exists)
            fileText.append("\"" + idFile.getClassificationText() + "\"");
            fileText.append(",");
            fileText.append("\"" + idFile.getFilePath() + "\"");
            fileText.append(",");
            fileText.append("\"" + idFile.getWarning() + "\"");

            fileText.append("\n"); //Write new line

            //Iterate through file format hits for the identification file
            for (hitCounter = 0; hitCounter < idFile.getNumHits(); hitCounter++) {
                hit = idFile.getHit(hitCounter);
                //First three columns are empty as these are for file info
                fileText.append(",,,");

                //Write the hit PUID,Format Name,Version,Hit Type and warning
                fileText.append("\"" + hit.getFileFormatPUID() + "\"");

                fileText.append(",");
                fileText.append("\"" + hit.getMimeType() + "\"");

                fileText.append(",");
                fileText.append("\"" + hit.getFileFormatName() + "\"");

                fileText.append(",");
                fileText.append("\"" + hit.getFileFormatVersion() + "\"");

                fileText.append(",");
                fileText.append("\"" + hit.getHitTypeVerbose() + "\"");

                fileText.append(",");
                fileText.append("\"" + hit.getHitWarning() + "\"");

                fileText.append("\n"); //Write new line
            }
        }

        //Write the text to specified file
        try {
            java.io.BufferedWriter out = new java.io.BufferedWriter(new java.io.FileWriter(filePath));
            out.write(fileText.toString());
            out.close();
        } catch (java.io.IOException e) {
            //TODO
        }
    }

    /**
     * Read in the default file collection file.
     */
    public void readFileCollection() throws Exception {
        readFileCollection(FILE_LIST_FILE_NAME);  //use default file name
    }

    /**
     * Read in a file collection file with the specified file name.
     *
     * @param theFileName Name of the file collection to read in
     */
    public void readFileCollection(String theFileName) {

        try {
            checkFile(theFileName);

            //prepare for XML read
            MessageDisplay.resetXMLRead();

            //prepare to read in the XML file
            SAXParserFactory factory = SAXParserFactory.newInstance();
            factory.setNamespaceAware(true);
            SAXParser saxParser = factory.newSAXParser();
            XMLReader parser = saxParser.getXMLReader();
            SAXModelBuilder mb = new SAXModelBuilder();
            mb.setObjectPackage("uk.gov.nationalarchives.droid");
            mb.setupNamespace(FILE_COLLECTION_NS, true);
            parser.setContentHandler(mb);

            //read in the XML file
            //specify the UTF8 encoding - otherwise will not interpret files with UTF8 characters correctly
            java.io.BufferedReader in = new java.io.BufferedReader(new java.io.InputStreamReader(new java.io.FileInputStream(theFileName), "UTF8"));
            parser.parse(new InputSource(in));
            myFileCollection = (FileCollection) mb.getModel();
            if (myFileCollection == null) {
                throw new Exception("The file " + theFileName + " doesn't specify any files.");
            }
            myFileCollection.setFileName(theFileName);

            //let the user know the outcome
            String successMessage = "The XML file " + theFileName;
            int numXMLWarnings = MessageDisplay.getNumXMLWarnings();
            if (numXMLWarnings == 0) {
                successMessage += " was successfully loaded";
            } else if (numXMLWarnings == 1) {
                successMessage += " was loaded with 1 warning";
            } else {
                successMessage += " was loaded with " + numXMLWarnings + " warnings";
            }
            int numLoadedFiles = myFileCollection.getNumFiles();
            if (numLoadedFiles == 0) {
                successMessage += "\nNo files were read in";
            } else if (numLoadedFiles == 1) {
                successMessage += "\n1 file was read in";
            } else {
                successMessage += "\n" + numLoadedFiles + " files were read in";
            }
            MessageDisplay.generalInformation(successMessage);

            int theFileSigFileVersion = myFileCollection.getLoadedFileSigFileVersion();
            int theCurrentSigFileVersion = getSigFileVersion();
            if ((theFileSigFileVersion > 0) && (theCurrentSigFileVersion != theFileSigFileVersion)) {
                MessageDisplay.generalWarning("The file was generated with signature file V" + theFileSigFileVersion + ".  The current signature file is V" + theCurrentSigFileVersion);
            }

        } catch (Exception e) {
            MessageDisplay.generalWarning(e.toString());
            myFileCollection = null;
        }
    }


    /**
     * Reads and parses the signature file that is pointed to in the configuration file
     */
    public void readSigFile() throws Exception {

        String theSigFileName = myConfigFile.getSigFileName();
        readSigFile(theSigFileName, true);
    }


    /**
     * Reads in and parses the signature file
     * Updates the configuration file with this signature file
     *
     * @param theSigFileName Name of the signature file
     */
    public String readSigFile(String theSigFileName) throws Exception {
        readSigFile(theSigFileName, true);
        return mySigFile.getVersion();
    }

    /**
     * Reads and parses the signature file
     *
     * @param theSigFileName Name of the signature file
     * @param isConfigSave   Flag to indicate whether the configuration file should be updated with this signature file.
     */
    public void readSigFile(String theSigFileName, boolean isConfigSave) throws Exception {
        readSigFile(theSigFileName, isConfigSave, false);
    }

    /**
     * Read the sigfile from a URL
     *
     * @param theSigFileURL
     * @return the sigfile version
     * @throws Exception
     */
    public String readSigFile(URL theSigFileURL) throws Exception {
        try {
            mySigFile = parseSigFile(theSigFileURL);

            mySigFile.prepareForUse();
            myConfigFile.setSigFileVersion(mySigFile.getVersion());
        } catch (Exception e) {
            throw e;
        }
        return mySigFile.getVersion();
    }

    /**
     * Reads and parses the signature file
     *
     * @param theSigFileName Name of the signature file
     * @param isConfigSave   Flag to indicate whether the configuration file should be updated with this signature file.
     * @param hideWarning    gives the ability to hide the warning if a file failes to load
     */
    public void readSigFile(String theSigFileName, boolean isConfigSave, boolean hideWarning) throws Exception {

        try {
            //checks that the file exists, throws an error if it doesn't
            checkFile(theSigFileName);

            //store the name of the new signature file
            myConfigFile.setSigFile(theSigFileName);

            //prepare for XML read
            MessageDisplay.resetXMLRead();

            //carry out XML read
            mySigFile = parseSigFile(theSigFileName);


            mySigFile.prepareForUse();

            String theVersion = mySigFile.getVersion();
            try {
                myConfigFile.setSigFileVersion(theVersion);
            } catch (Exception e) {
                e.printStackTrace();
            }

            //let the user know the outcome
            MessageDisplay.setStatusText("Current signature file is V" + getSigFileVersion(), "Signature file V" + getSigFileVersion() + " has been checked");
            int numXMLWarnings = MessageDisplay.getNumXMLWarnings();
            if (numXMLWarnings > 0) {
                String successMessage = "The signature file " + theSigFileName + " was loaded with " + numXMLWarnings + " warnings";
                String cmdlineMessage = numXMLWarnings + " warnings were found";
                MessageDisplay.generalInformation(successMessage, cmdlineMessage);
            }

            if (isConfigSave) {
                //update the configuration file to contain the details of this signature file
                try {
                    saveConfiguration();
                } catch (IOException e) {
                    MessageDisplay.generalWarning("Unable to save configuration updates");
                }
            }

        } catch (Exception e) {
            if (!hideWarning) {
                MessageDisplay.generalWarning(e.toString());
            }
            mySigFile = null;
        }

    }


    /**
     * Create the XML parser for the signature file
     *
     * @return XMLReader
     * @throws Exception
     */
    private XMLReader getXMLReader(SAXModelBuilder mb) throws Exception {

        SAXParserFactory factory = SAXParserFactory.newInstance();
        factory.setNamespaceAware(true);
        SAXParser saxParser = factory.newSAXParser();
        XMLReader parser = saxParser.getXMLReader();
        mb.setupNamespace(SIGNATURE_FILE_NS, true);
        parser.setContentHandler(mb);
        return parser;
    }

    /**
     * parse the sig file as a URL
     *
     * @param signatureFileURL
     * @return
     * @throws Exception
     */
    public FFSignatureFile parseSigFile(URL signatureFileURL) throws Exception {

        SAXModelBuilder mb = new SAXModelBuilder();
        XMLReader parser = getXMLReader(mb);

        //read in the XML file
        java.io.BufferedReader in = new java.io.BufferedReader(new java.io.InputStreamReader(signatureFileURL.openStream(), "UTF8"));
        parser.parse(new InputSource(in));
        return (FFSignatureFile) mb.getModel();
    }


    /**
     * Create a new signature file object based on a signature file
     *
     * @param theFileName the file name
     */
    public FFSignatureFile parseSigFile(String theFileName) throws Exception {

        SAXModelBuilder mb = new SAXModelBuilder();
        XMLReader parser = getXMLReader(mb);

        //read in the XML file
        java.io.BufferedReader in = new java.io.BufferedReader(new java.io.InputStreamReader(new FileInputStream(theFileName), "UTF8"));
        parser.parse(new InputSource(in));
        return (FFSignatureFile) mb.getModel();
    }


    /**
     * Checks whether the signature file has been loaded.  If it hasn't, then
     * either exits the application or downloads a new signature file from
     * PRONOM web service.
     */
    public void checkSignatureFile() {

        if (mySigFile == null) {
            if (MessageDisplay.exitDueToMissigSigFile()) {
                System.exit(0);
            } else {
                this.downloadwwwSigFile();
                if (!PronomWebService.isCommSuccess) {
                    //failed to download signature file
                    //Message to warn user
                    String failureMessage = "Unable to connect to the PRONOM web service. Make sure that the following settings in your configuration file (DROID_config.xml) are correct:\n";
                    failureMessage += "    1- <SigFileURL> is the URL of the PRONOM web service.  This should be '" + AnalysisController.PRONOM_WEB_SERVICE_URL + "'\n";
                    failureMessage += "    2- <ProxyHost> is the IP address of the proxy server if one is required\n";
                    failureMessage += "    3- <ProxyPort> is the port to use on the proxy server if one is required\n\n";
                    failureMessage += "Droid will now close down.";
                    //Warn the user that the connection failed
                    MessageDisplay.fatalError(failureMessage);
                    //javax.swing.JOptionPane.showMessageDialog(this,failureMessage,"Web service connection error",javax.swing.JOptionPane.WARNING_MESSAGE) ;            

                    System.exit(0);
                }
            }
        }
    }

    /**
     * Checks that a file name corresponds to a file that exists and can be opened
     *
     * @param theFileName file name
     */
    private void checkFile(String theFileName) throws Exception {

        java.io.File file = new java.io.File(theFileName);
        if (!file.exists()) {
            throw new Exception("The file " + theFileName + " does not exist");
        } else if (!file.canRead()) {
            throw new Exception("The file " + theFileName + " cannot be read");
        } else if (file.isDirectory()) {
            throw new Exception("The file " + theFileName + " is a directory");
        }
    }

    /**
     * Check whether a file exists
     *
     * @param theFileName The full path of the file to check
     */
    public boolean isFileFound(String theFileName) {
        java.io.File file = new java.io.File(theFileName);
        if (file.exists()) {
            return true;
        } else {
            return false;
        }
    }

    /**
     * Empties the file list
     */
    public void resetFileList() {
        myFileCollection.removeAll();
    }

    /**
     * Add files to list of files ready for identification.
     * Calls addFile(fileFolderName, false)
     *
     * @param fileFolderName file or folder to add to
     */
    public void addFile(String fileFolderName) {
        addFile(fileFolderName, false);
    }

    /**
     * Add file to list of files ready for identification.
     * If the file is already in list, then does not add it.
     * If the file is a folder, then adds all files it contains.
     * If isRecursive is set to true, then it also searches recursively through any subfolders.
     *
     * @param fileFolderName file or folder to add to
     * @param isRecursive    whether or not to search folders recursively
     */
    public void addFile(String fileFolderName, boolean isRecursive) {
        myFileCollection.addFile(fileFolderName, isRecursive);
    }

    /**
     * Remove file from the file list
     *
     * @param theFileName the name of the file to remove
     */
    public void removeFile(String theFileName) {
        myFileCollection.removeFile(theFileName);
    }

    public void removeFile(int theIndex) {
        myFileCollection.removeFile(theIndex);
    }

    /**
     * Returns an identificationFile object based on its index in the list
     *
     * @param theIndex index of file in file collection
     */
    public IdentificationFile getFile(int theIndex) {
        IdentificationFile theFile = null;
        try {
            theFile = myFileCollection.getFile(theIndex);
            //return myFileCollection.getFile(theIndex);
        } catch (Exception e) {
        }
        return theFile;
    }

    /**
     * Returns the number of files in identification file list
     */
    public int getNumFiles() {
        int theNumFiles = 0;
        try {
            theNumFiles = myFileCollection.getNumFiles();
        } catch (Exception e) {
        }
        return theNumFiles;
        //return myFileCollection.getNumFiles() ;
    }
    
    /**
     * Get an iterator for the file collection
     */
    public Iterator<IdentificationFile> getFileIterator(){
        return this.myFileCollection.getIterator();
    }

    /**
     * Return the version of the currently loaded signature file
     */
    public int getSigFileVersion() {
        int theVersion = 0;
        try {
            theVersion = Integer.parseInt(mySigFile.getVersion());
        } catch (Exception e) {
        }
        return theVersion;
        //return Integer.parseInt(mySigFile.getVersion());
    }

    public FFSignatureFile getSigFile() {
        return mySigFile;
    }

    /**
     * Return the current proxy host setting
     */
    public String getProxyHost() {
        return myConfigFile.getProxyHost();
    }

    /**
     * Set the proxy host
     */
    public void setProxyHost(String value) {
        myConfigFile.setProxyHost(value);
    }

    /**
     * Return the current proxy port setting
     */
    public int getProxyPort() {
        return myConfigFile.getProxyPort();
    }

    /**
     * Set the proxy port number
     */
    public void setProxyPort(int value) {
        myConfigFile.setProxyPort(Integer.toString(value));
    }

    /**
     * Return the version of the uk application
     */
    public static String getDROIDVersion() {
        String theVersion = DROID_VERSION;

        //remove number after last .  This is a development version, not to be displayed in About box
        int theLastDot = theVersion.lastIndexOf(".");
        if (theLastDot > -1) {
            if (theVersion.indexOf(".") < theLastDot) {
                theVersion = theVersion.substring(0, theLastDot);
            }
        }
        return theVersion;
    }

    /**
     * Returns the number of files that have been analysed.
     */
    public int getNumCompletedFiles() {
        if (isAnalysisRunning) {
            return myNumCompletedFiles;
        } else {
            int theNumCompletedFiles = 0;
            java.util.Iterator<IdentificationFile> files = myFileCollection.getIterator();
            while (files.hasNext()){
                if (files.next().isClassified()){
                    theNumCompletedFiles++;
                }
            }
            return theNumCompletedFiles;
        }

    }
    
    /**
     * Get the time that analysis or profiling started
     * @return date and time started
     */
    public Date getStartTime(){
        return this.startTime;
    }
    
    /**
     * Get the time that analysis or profiling completed
     * @return date and time completed
     */
    public Date getCompletedTime(){
        return this.completedTime;
    }


    /**
     * Records the fact that analysis has finished and saves to file if a request has been made
     */
    public void setAnalysisComplete() {
        isAnalysisRunning = false;
        this.completedTime = new Date();
        
        //save results if requested
        if (myOutFormats.indexOf("XML") > -1 || myOutFormats.indexOf("xml") > -1) {
            saveFileList(myOutFileName + ".xml", true);
        }
        if (myOutFormats.indexOf("CSV") > -1 || myOutFormats.indexOf("csv") > -1) {
            exportFileCollectionAsCSV(myOutFileName + ".csv");
        }

    }

    /** 
     * Export statistics results with filename / format given at the command line
     */
    public void commandLineStatsExport(){

            String[] formats = myOutFormats.split(",");
            boolean exportSuccess;
            String path;
            
            for (String format : formats){
                try {
                    // Create create exporter for the format
                    StatsExportFormat exporter = StatsExporterFactory.createExportFormat(format);
                    
                    // Compile file path
                    path = myOutFileName + "." + format;
                    
                    // Perform export
                    exportSuccess = exporter.export(getStatsLogger(), this.getStartTime(), this.getCompletedTime(), path);
                    
                    if (exportSuccess){
                        System.out.println("Created "+format.toUpperCase()+" file "+path);
                    }else{
                        System.out.println("Failed to create "+format.toUpperCase()+" file. "+exporter.getCompilationError());
                    }
                } catch (StatsExporterFactory.FormatUnknownException e){
                    System.out.println("Unknown export format: " + format);
                }
            }
            
    }
    
    /**
     * Set flag to mark analysis as complete.
     */
    public void setStatsComplete() {
        isAnalysisRunning = false;
        boolean success = false;
        this.completedTime = new Date();
        //Save file if user has chosen a file (command line)
        if (myOutFormats.length() > 0){
            commandLineStatsExport();
        }
            
    }
    
    /**
     * Checks whether analysis has finished yet
     */
    public boolean isAnalysisComplete() {
        return !isAnalysisRunning;
    }

    /**
     * Record start of anlaysis
     */
    public void setAnalysisStart() {
        myNumCompletedFiles = 0;
        isAnalysisRunning = true;
        this.startTime = new Date();
    }

    public void setLogger(StatsLogger logger) {
        this.myLogger = logger;
    }
    /**
     * Records the fact that a file has been anlaysed
     */
    public void incrNumCompletedFile() {
        myNumCompletedFiles++;
    }


    /**
     * Checks whether analysis has been cancelled by the user
     */
    public boolean isAnalysisCancelled() {
        return myAnalysisCancelled;
    }

    /**
     * Cancel the analysis.  This will cause it to stop as soon as it has finished
     * the file it is working on.
     */
    public void cancelAnalysis() {
        myAnalysisCancelled = true;
    }

    /**
     * Launch the analysis thread on the files that have been listed and using
     * the signature file that has been opened.
     */
    public void runFileFormatAnalysis() {        
        myAnalysisCancelled = false;
        myOutFileName = "";
        myOutFormats = "";
        new AnalysisThread(myFileCollection, mySigFile, this).start();
    }

    /**
     * Launch the analysis thread on the files that have been listed and using
     * the signature file that has been opened.  Save results to file at the end of the run.
     *
     * @param theOutFormats  string containing formats for the output (code looks for CSV and XML in the string)
     * @param theOutFileName name of file to which to save results at end of run
     */
    public void runFileFormatAnalysis(String theOutFormats, String theOutFileName) {       
        myAnalysisCancelled = false;
        myOutFileName = theOutFileName;
        myOutFormats = theOutFormats;
        new AnalysisThread(myFileCollection, mySigFile, this).start();
    }

    /**
     * Access to the file collection
     *
     * @return the current file collection
     */
    public FileCollection getFileCollection() {
        return myFileCollection;
    }

    /**
     * checks whether there is a signature file available through the PRONOM web service
     * which is a later version than the one currently loaded.
     */
    public boolean isNewerSigFileAvailable() {
    	return isNewerSigFileAvailable(this.getSigFileVersion());
    }
    
    /**
     * checks whether there is a signature file available through the PRONOM web service
     * which is a later version than the specified version number.
     * 
     * @param currentVersion
     * @return
     */
    public boolean isNewerSigFileAvailable(int currentVersion){
        int theLatestVersion;
        try {
        	Element versionXML = PronomWebService.sendRequest(myConfigFile.getSigFileURL(), myConfigFile.getProxyHost(), myConfigFile.getProxyPort(), "getSignatureFileVersionV1", null);
        	Boolean deprecated = Boolean.parseBoolean(PronomWebService.extractXMLelement(versionXML, "Deprecated").getValue());
        	if( deprecated ) {
        		String message = "A new version of DROID is available.\nPlease visit http://droid.sourceforge.net";
        		MessageDisplay.generalInformation(message);
        	}
        	theLatestVersion = Integer.parseInt(PronomWebService.extractXMLelement(versionXML, "Version").getValue());
            sigFileLatestVersion = theLatestVersion;
            MessageDisplay.setStatusText("The latest signature file available is V" + sigFileLatestVersion);
        } catch (Exception e) {
            MessageDisplay.generalWarning("Unable to get signature file version from PRONOM website:\n" + e.getMessage());
            return false;
        }
        return (theLatestVersion > currentVersion);
        
    }

    /**
     * Download the latest signature file from the PRONOM web service, save it to file
     * An input flag determines whether or not to load it in to the current instance of uk
     *
     * @param theFileName   file where to save signature file
     * @param isLoadSigFile Flag indicating whether to load the signature file into the current instance of uk
     */
    public void downloadwwwSigFile(String theFileName, boolean isLoadSigFile) {
        try {
        	Element sigFile = PronomWebService.sendRequest(myConfigFile.getSigFileURL(), myConfigFile.getProxyHost(), myConfigFile.getProxyPort(), "getSignatureFileV1", "FFSignatureFile");
        	try {
        		XMLOutputter outputter = new XMLOutputter();
        		java.io.BufferedWriter out = new java.io.BufferedWriter(new java.io.FileWriter(theFileName));
        		out.write("<?xml version=\"1.0\" encoding=\"UTF-8\"?>");
                outputter.output(sigFile, out);
                out.close();
                if (isLoadSigFile) {
                    try {
                        myConfigFile.setDateLastDownload();
                        readSigFile(theFileName);
                    } catch (Exception e) {
                        MessageDisplay.generalWarning("Unable to read in downloaded signature file");
                    }
                }
            } catch (Exception e) {
                MessageDisplay.generalWarning("Unable to save downloaded signature file");
            }

        } catch (Exception e) {
            MessageDisplay.generalWarning("Unable to download signature file from the PRONOM web service");
        }

    }

    /**
     * Download the latest signature file from the PRONOM web service, save it to file
     * and load it in to the current instance of uk
     *
     * @param theFileName file where to save signature file
     */
    public void downloadwwwSigFile(String theFileName) {
        downloadwwwSigFile(theFileName, true);
    }

    /**
     * Download the latest signature file from the PRONOM web service, save it to a DEFAULT location
     * and load it in to the current instance of uk
     * Saves to same folder as current signature file but as
     * DROID_Signature_V[X].xml , where [X] is the version number of the signature file
     */
    public void downloadwwwSigFile() {

        try {
            int theLatestVersion = Integer.parseInt(PronomWebService.sendRequest(myConfigFile.getSigFileURL(), myConfigFile.getProxyHost(), myConfigFile.getProxyPort(), "getSignatureFileVersionV1", "Version").getValue());

            java.io.File currentSigFile = new java.io.File(myConfigFile.getSigFileName());

            String currentPath = "";

            if (currentSigFile != null) {
                currentPath = (currentSigFile.getAbsoluteFile()).getParent() + java.io.File.separator;
            }

            final String newSigFileName = currentPath
                    + "DROID_SignatureFile_V"
                    + theLatestVersion
                    + ".xml";

            /*
        //Setup save file dialog
        javax.swing.JFileChooser fc = new javax.swing.JFileChooser() ;
        fc.addChoosableFileFilter(new uk.GUI.CustomFileFilter(FILE_COLLECTION_FILE_EXTENSTION,FILE_COLLECTION_FILE_DESCRIPTION));
        fc.setAcceptAllFileFilterUsed(false);
        fc.setDialogTitle("Save" + dialogTitle) ;
        //show file dialog
        int returnVal = fc.showSaveDialog(this);
             
        //Save file if user has chosen a file
        if (returnVal==javax.swing.JFileChooser.APPROVE_OPTION){
             
          //Get file user selected
           path = fc.getSelectedFile();
             
           //if no extension was specified add one
           if(!(path.getName().endsWith("." + FILE_COLLECTION_FILE_EXTENSTION))){
                path = new java.io.File(path.getParentFile(), path.getName() + "." + FILE_COLLECTION_FILE_EXTENSTION);
            }
             
           //if path exists check confirm with user if they want to overwrite
           if(path.exists()){
                int option = javax.swing.JOptionPane.showConfirmDialog(this, "The specified file exists, overwrite?");
                if(option != javax.swing.JOptionPane.YES_OPTION)return;
            }*/

            downloadwwwSigFile(newSigFileName);
        } catch (Exception e) {
            MessageDisplay.generalWarning("Unable to download signature file from the PRONOM web service");
        }

    }

    /**
     * Checks whether a new signature file download is due
     * based on current date and settings in the configuration file
     */
    public boolean isSigFileDownloadDue() {

        return myConfigFile.isDownloadDue();
    }

    /**
     * Returns the date current signature file was created
     *
     * @return date signature file was created
     */
    public String getSignatureFileDate() {
        String theDate = "";
        try {
            theDate = mySigFile.getDateCreated();
        } catch (Exception e) {
        }
        if (theDate.equals("")) {
            theDate = "No date given";
        }
        return theDate;
        //return mySigFile.getDateCreated() ;
    }

    /**
     * Returns the file path of the signature file
     *
     * @return signature file file path
     */
    public String getSignatureFileName() {
        return myConfigFile.getSigFileName();
    }
    
    /**
     * Return the statistics logger object
     */
    public StatsLogger getStatsLogger() {
       return this.myLogger; 
    }

    /**
     * Get the PUID resolution base URL
     *
     * @return
     */
    public String getPuidResolutionURL() {
        return myConfigFile.getPuidResolution();
    }
    
    /**
     * Sets the URL of the signature file webservices
     * 
     * @param sigFileURL
     */
    public void setSigFileURL(String sigFileURL) {
    	myConfigFile.setSigFileURL(sigFileURL);
    }

    /**
     * Get the browser binary path
     */
    public String getBrowserPath() {
        return myConfigFile.getBrowserPath();
    }

    /**
     * Gets the number of days after which user should be alerted for new signature file
     *
     * @return number of days after which user should be alerted for new signature file
     */
    public int getSigFileCheckFreq() {
        return myConfigFile.getSigFileCheckFreq();
    }

    /**
     * Updates the configuration parameter which records the interval after which
     * the signature file should be updated.
     *
     * @param theFreq The number of days after which the user will be prompted to check for a newer signature file
     */
    public void setSigFileCheckFreq(String theFreq) {
        myConfigFile.setSigFileCheckFreq(theFreq);
    }

    /**
     * updates the DateLastDownload element of the configuration file and updates
     * the configuration file.  This is to be used whenever the user checks for
     * a signature file update, but one is not found
     */
    public void updateDateLastDownload() {
        //set the DateLastDownload to now
        myConfigFile.setDateLastDownload();
        //save to file
        try {
            saveConfiguration();
        } catch (IOException e) {
            MessageDisplay.generalWarning("Unable to save configuration updates");
        }
    }

    /**
     * Returns the verbosity for command line out
     * 
     * @return
     */
	public boolean isVerbose() {
		return verbose;
	}

	/**
	 * Set the command line verbosity
	 * 
	 * @param verbose
	 */
	public void setVerbose(boolean verbose) {
		this.verbose = verbose;
	}


}

